# -*- coding: utf-8 -*-

import newshelper
import util
from crawler import Crawler


class TouTiaoNews(Crawler):

    def download(self, params):
        url = 'http://toutiao.com/search_content'

        retval = []

        for x in xrange(12):
            content = self._fetch_content(url, {
                'offset': 20 * x,
                'format': 'json',
                'keyword': params['keyword'],
                'autoload': 'true',
                'count': 20
            })
            if not content:
                continue

            retval.append(content)

        return retval

    def _crawl(self, params):
        url = 'http://toutiao.com/search_content'

        retval = []

        for x in xrange(12):
            content = self._fetch_content(url, {
                'offset': 20 * x,
                'format': 'json',
                'keyword': params['keyword'],
                'autoload': 'true',
                'count': 20
            })
            if not content:
                continue

            content = self._structuring_content(content)
            if not content:
                continue

            content = content.find('body').find('p').get_text()
            if not content:
                continue

            content = eval(self._replace_content([('false', 'False'), ('true', 'True')], content))

            for x in content['data']:
                if params['time_limit'] > x['datetime']:
                    continue

                if not params.get('company_name', ''):
                    news = newshelper.purify_and_format(
                        x, params['keyword'], self._fetch_and_extract_content(x['url']), 'toutiao'
                    )
                    self._logger.debug('formatted news: %s' % util.ustr(news))

                    retval.append(news)

        return retval
